﻿using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using log4net;
using Newtonsoft.Json;
using org.apache.pdfbox.pdmodel;
using org.apache.pdfbox.util;

namespace Spider.Spider
{
    public class CompanyNoticeSpider
    {

        public static readonly Dictionary<string, string> CookieDictionary = new Dictionary<string, string>();

        public static string GetFilePath(string id)
        {
            return GetFileDirPath() + id + ".pdf";
        }
        public static string GetTempFilePath(string id)
        {
            return GetTempDirPath() + id;
        }

        public static string GetHtmlFilePath(string id)
        {
            return GetTempDirPath() + id;
        }

        public static string GetFileDirPath()
        {
            return @"D:\hot_topic\company_notice\";
        }

        public static string GetTempDirPath()
        {
            return @"D:\hot_topic\temp\";
        }
       

        public static bool HttpDownload(string id, string url)
        {
            string path = GetFilePath(id);
            System.IO.Directory.CreateDirectory(GetTempDirPath());  //创建临时文件目录
            System.IO.Directory.CreateDirectory(GetFileDirPath());  //创建文件目录

            string tempFile = GetTempFilePath(id);
            if (System.IO.File.Exists(tempFile))
            {
                System.IO.File.Delete(tempFile);    //存在则删除
            }
            try
            {
                FileStream fs = new FileStream(tempFile, FileMode.Append, FileAccess.Write, FileShare.ReadWrite);
                // 设置参数
                HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
                //发送请求并获取相应回应数据
                HttpWebResponse response = request.GetResponse() as HttpWebResponse;
                //直到request.GetResponse()程序才开始向目标网页发送Post请求
                Stream responseStream = response.GetResponseStream();
                //创建本地文件写入流
                //Stream stream = new FileStream(tempFile, FileMode.Create);
                byte[] bArr = new byte[1024];
                int size = responseStream.Read(bArr, 0, (int)bArr.Length);
                while (size > 0)
                {
                    //stream.Write(bArr, 0, size);
                    fs.Write(bArr, 0, size);
                    size = responseStream.Read(bArr, 0, (int)bArr.Length);
                }
                //stream.Close();
                fs.Close();
                responseStream.Close();
                System.IO.File.Move(tempFile, path);
                return true;
            }
            catch (Exception ex)
            {
                return false;
            }
        }

        private static readonly ILog Log = LogManager.GetLogger(typeof(CompanyNoticeSpider));
        
        /// <summary>
        /// 取多少号之后的
        /// </summary>
        /// <param name="startTime"></param>
        /// <param name="isall"></param>
        /// <returns></returns>
        public static List<Announcements> GetPubnote(DateTime startTime,bool isall)
        {
            Log.Info("开始获取公告");
            //string noticetime = RedisControler.GetRedisData("NoticeTime");
            BlockingCollection<Announcements> notice = new BlockingCollection<Announcements>();

            var tasks = new List<Task>();

            for (int i = 0; startTime.AddDays(i).Date <= DateTime.Now.Date; i++)
            {
                var dateTime = startTime.AddDays(i);
                var startNew = Task.Factory.StartNew(() =>
                {
                    var list = GetPubnote(dateTime);

                    foreach (var announcementse in list)
                    {
                        if ((File.Exists(GetFilePath(announcementse.announcementId))))
                        {
                            try
                            {
                                PDDocument doc = PDDocument.load(GetFilePath(announcementse.announcementId));
                                try
                                {
                                    PDFTextStripper pdfStripper = new PDFTextStripper();
                                    announcementse.announcementContent = pdfStripper.getText(doc);
                                    doc.close();
                                }
                                catch (Exception e)
                                {
                                    doc.close();
                                    Log.Error(e);
                                }
                            }
                            catch (Exception e)
                            {
                                Log.Error(e);
                            }
                            continue;
                        }

                        var httpDownload = HttpDownload(announcementse.announcementId, announcementse.adjunctUrl);
                        if (httpDownload)
                        {
                            try
                            {
                                PDDocument doc = PDDocument.load(GetFilePath(announcementse.announcementId));
                                try
                                {
                                    PDFTextStripper pdfStripper = new PDFTextStripper();
                                    announcementse.announcementContent = pdfStripper.getText(doc);
                                    doc.close();
                                }
                                catch (Exception e)
                                {
                                    doc.close();
                                    Log.Error(e);
                                }
                            }
                            catch (Exception e)
                            {
                                Log.Error(e);
                            }
                        }
                    }

                    list.ForEach(m=>notice.Add(m));
                });
                tasks.Add(startNew);
            }
            Task.WaitAll(tasks.ToArray());
            var announcementses = new List<Announcements>();
            announcementses.AddRange(notice);
            announcementses = announcementses.Where(m => m.time > startTime).ToList();
            Log.Info("获取完毕");
            return announcementses;
        }
        
        public static List<Announcements> GetPubnote(DateTime startTime)
        {
            var time = startTime.ToString("yyyy-MM-dd");
            Log.Info("爬取公告时间：" + time);
            var date = time;
            var url = "http://www.cninfo.com.cn/new/hisAnnouncement/query";
            var data = $"column=sse&columnTitle=历史公告查询" +
                       $"&pageNum={1}" +
                       $"&pageSize=50&tabName=fulltext" +
                       $"&seDate={date}";
            var responseJson = Controller.PostData(url, data);
            //RedisClient8.Set("test", responseJson);
            var responseDatas = Controller.JsonToObject<CnInfoResponse>(responseJson);
            if (responseDatas.announcements == null) return null; //假期搜索功能一般返回空值
            //var responseDatas = RedisClient8.Get<CnInfoResponse>("test");
            var pageCount = Convert.ToInt32(responseDatas.totalRecordNum) / 50 + 1;
            
            var list = new List<Announcements>();

            for (var i = 1; i <= pageCount; i++)
            {
                list = list.Concat(GetPubnote(date, i)).ToList();
                //Log.Info($"{DateTime.Now}|{date}|page {i}");
            }
            return list;
        }
        
        private static List<Announcements> GetPubnote(string date, int pageNum)
        {
            try
            {
                var url = "http://www.cninfo.com.cn/new/hisAnnouncement/query";
                var data = $"column=sse&columnTitle=历史公告查询" +
                           $"&pageNum={pageNum}" +
                           $"&pageSize=50&tabName=fulltext" +
                           $"&seDate={date}";
                var responseJson = Controller.PostData(url, data);
                var responseDatas = Controller.JsonToObject<CnInfoResponse>(responseJson);
                return responseDatas.announcements;
            }
            catch (Exception e)
            {
                Log.Error(e);
            }

            return new List<Announcements>();
        }

    }

    public class CnInfoResponse
    {
        public List<Announcements> announcements { get; set; }
        public string categoryList { get; set; }
        public string classifiedAnnouncements { get; set; }
        public bool hasMore { get; set; }
        public string totalAnnouncement { get; set; }
        public string totalRecordNum { get; set; }
        public string totalSecurities { get; set; }
    }

    public class Announcements
    {
        public string adjunctSize { get; set; }
        public string adjunctType { get; set; }

        private string adjuncturl;
        public string adjunctUrl { get { return "http://www.cninfo.com.cn/" + adjuncturl; } set { adjuncturl = value; } }
        public string announcementContent { get; set; }
        public string announcementId { get; set; }

        private string announcementtime;

        public string announcementTime { get { return StampToDateTime(announcementtime).ToString(); } set { announcementtime = value; } }

        public string announcementTitle { get; set; }
        public string announcementType { get; set; }
        public string announcementTypeName { get; set; }
        public string associateAnnouncement { get; set; }
        public string batchNum { get; set; }
        public string columnId { get; set; }
        public string id { get; set; }
        public string important { get; set; }
        public string orgId { get; set; }
        public string pageColumn { get; set; }

        //    /// <summary>
        //    /// 股票代码
        //    /// </summary>
        string seccode;
        public string secCode
        {
            get { return seccode.Substring(0, 1) == "6" ? seccode + ".SH" : seccode + ".SZ"; }
            set { seccode = value; }
        }
        public string secName { get; set; }
        public string storageTime { get; set; }
        public DateTime time=> StampToDateTime(announcementtime);

        // 时间戳转为C#格式时间
        private DateTime StampToDateTime(string timeStamp)
        {
            timeStamp = timeStamp.Remove(timeStamp.Length - 3, 3);
            DateTime dateTimeStart = TimeZone.CurrentTimeZone.ToLocalTime(new DateTime(1970, 1, 1));
            long lTime = long.Parse(timeStamp + "0000000");
            TimeSpan toNow = new TimeSpan(lTime);

            return dateTimeStart.Add(toNow);
        }

        // DateTime时间格式转换为Unix时间戳格式
        private int DateTimeToStamp(System.DateTime time)
        {
            System.DateTime startTime = TimeZone.CurrentTimeZone.ToLocalTime(new System.DateTime(1970, 1, 1));
            return (int)(time - startTime).TotalSeconds;
        }

    }

    public class Controller
    {
        #region 对象JSON化

        /// <summary>
        /// 把object对象JSON化
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="obj"></param>
        /// <returns></returns>
        public static string ObjectToJson<T>(T obj)
        {
            return JsonConvert.SerializeObject(obj);
        }

        #endregion

        #region 还原Redis中的JSON化对象

        /// <summary>
        /// 还原Redis中的JSON化对象
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="json"></param>
        /// <returns></returns>
        public static T JsonToObject<T>(string json)
        {
            if (json == null)
                json = ObjectToJson(default(T));
            return JsonConvert.DeserializeObject<T>(json);
        }

        #endregion

        public static string PostData(string url, string postData)
        {
            ASCIIEncoding encoding = new ASCIIEncoding();
            byte[] data = encoding.GetBytes(postData);
            HttpWebRequest myRequest = (HttpWebRequest) WebRequest.Create(url);

            myRequest.Method = "POST";
            myRequest.ContentType = "application/x-www-form-urlencoded";
            myRequest.ContentLength = data.Length;

            Stream newStream = myRequest.GetRequestStream();

            newStream.Write(data, 0, data.Length);
            newStream.Close();

            HttpWebResponse myResponse = (HttpWebResponse) myRequest.GetResponse();
            StreamReader reader = new StreamReader(myResponse.GetResponseStream(), Encoding.UTF8);
            string content = reader.ReadToEnd();
            reader.Close();

            return content;
        }
        
    }
}
